package inseniarlingua.tokenizer;

import java.util.ArrayList;
import java.util.List;

public class TokenizerBase {
	private String text;
	private int ipos;
	
	static final int BEG_OF_SENTENCE = 1;
	static final int BEETWEEN_WORDS = 2;
	
	public void setText(String text) {
		this.text = text;
	}
	
	public String getText() {
		return text;
	}
	
	public static String joinWords(List<?> list, int ibeg, int iend) {
		StringBuilder sb = new StringBuilder();
		for (int i=ibeg; i<iend; i++) {
			Object o = list.get(i);
			if (o == null) continue;
			if (sb.length() > 0) sb.append(' ');
			sb.append(o.toString());
		}
		return sb.toString();
	}
	
	private Word doWord() {
		int ibeg = ipos;
		int iend = ipos;
		while (ipos < text.length()) {
			char ch = text.charAt(ipos);
			if (Character.isLetterOrDigit(ch)) {
				iend = ipos+1;
			}
			else {
				ipos--;
				break;
			}
			ipos++;
		}
		String word = text.substring(ibeg, iend);
		Word tok = new Word(word);
		return tok;
	}
	
	public List<Word> tokenize() {
		ipos = 0;
		List<Word> ret = new ArrayList<Word>();
		Word token = null;
		while (ipos < text.length()) {
			int state = BEG_OF_SENTENCE;
			char ch = text.charAt(ipos);
			switch (state) {
			case BEG_OF_SENTENCE:
				if (Character.isLetter(ch)) {
					token = doWord();
				}
				else if ("',.!".indexOf(ch) >= 0) {
					token = new Word(ch+"");
				}
				if (token != null) {
					ret.add(token);
					token = null;
				}
				break;
			}
			ipos++;
		}
		return ret;
	}
}
